package com.discuz.spider.jpkoo;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;

import org.apache.commons.io.FileUtils;

import com.discuz.spider.WebClientAgent;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class Article {

	public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException, InterruptedException {
		
		getContent();
	    
	}
	
	public static void getAllListPage() throws FailingHttpStatusCodeException, MalformedURLException, IOException, InterruptedException{
		
		int sum =198;
		int index = 133;
		
		while(index<sum){
			
			String u = "http://free.activeclub.net/script/forum/list.asp?board_id=472956&Page="+index;
			
			HtmlPage p = WebClientAgent.get().getPage(u);
	        String b ="http://free.activeclub.net/script/forum/";
		    @SuppressWarnings("unchecked")
			List<HtmlAnchor> l = (List<HtmlAnchor>)p.getByXPath("html/body/form/table[3]/tbody/tr/td[1]/table/tbody/tr/td[2]/a");
		    for(HtmlAnchor h :l){
		    	String link = b + h.getAttribute("href");
		    	String title = h.asText();
		    	String r = title + "|" + link +"\n";
		    	FileUtils.writeStringToFile(new File("d:/list.txt"),r,true);
		    }
			System.out.println("current ...."+index);
			index++;
			Thread.sleep(3000);
		}
		
	
        
	}
	
	public  static void getContent() throws IOException {
		
		List<String> lines = FileUtils.readLines(new File("d:/list.txt"));
		int index = 0;
		int statr = 101;
		for(String line : lines){
			index++;
			if(index<statr) continue;
			try {
				String [] e = line.split("\\|");
				if(e.length == 2){
					HtmlPage p = WebClientAgent.get().getPage(e[1]);
					HtmlElement el = p.getFirstByXPath("//*[@class='ArticleContent']");
					if(el == null){
						
						System.out.println("no----"+index);
						continue;
					}
					String c = el.asText();
					FileUtils.writeStringToFile(new File("d:/data/"+e[0]+".txt"),c);
					System.out.println("----"+index);
					Thread.sleep(2000);
				}
			} catch (FailingHttpStatusCodeException e) {
				e.printStackTrace();
				break;
			} catch (MalformedURLException e) {
				e.printStackTrace();
				break;
			} catch (IOException e) {
				e.printStackTrace();
				break;
			} catch (InterruptedException e) {
				e.printStackTrace();
				break;
			}
		}
		
		
	}

}
